// Released to the public domain. Use, modify and relicense at will.

// F# Demonstration Script: Web Crawling
//
// This script is design for use with F# Interactive,
//
// Highlight the code and send it to F# Interactive,
// e.g. using Alt-Enter from Visual Studio. Start F# Interactive using 
// View -> Other Windows -> F# Interactive.


//---------------------------------------------------------------------------
// Part O. Hello World. 

System.Console.WriteLine("Hello World")

//---------------------------------------------------------------------------
// Part I. Web.

open System.Net
open System
open System.IO

// ok, now fetch a page.  Create the web request,
// wait for the response, read off the response.
let req = System.Net.WebRequest.Create("http://www.yahoo.com")
let stream = req.GetResponse().GetResponseStream()
let reader = new IO.StreamReader(stream)
let html = reader.ReadToEnd()

// Ok, let's lok at the HTML
html

// If the above failed, you may need to set up your web proxy to your system default
//System.Net.WebRequest.DefaultWebProxy <- System.Net.WebRequest.GetSystemWebProxy()


/// Fetch the contents of a web page
let http(url: string) = 
    let req = System.Net.WebRequest.Create(url) 
    let resp = req.GetResponse()
    let stream = resp.GetResponseStream() 
    let reader = new IO.StreamReader(stream) 
    let html = reader.ReadToEnd()
    resp.Close()
    html

let live   = http("http://www.live.com")
let google = http("http://www.google.com")
let bbc    = http("http://news.bbc.co.uk")
let msft   = http("http://www.microsoft.com")
let nytRSS = http("http://www.nytimes.com/services/xml/rss/nyt/HomePage.xml")

nytRSS

// ----------------------------
// Windows Forms


open System.Windows.Forms
open System.Drawing

let form = new Form(Visible = true, Text = "A Simple F# Form", 
                    TopMost = true, Size = Size(600,600))

let textB = new RichTextBox(Dock = DockStyle.Fill, Text = "F# Programming is Fun!",
                            Font = new Font("Lucida Console",16.0f,FontStyle.Bold),
                            ForeColor = Color.DarkBlue)


form.Controls.Add(textB)

textB.Click.Add(fun evArgs -> printf "Click!\n")


// ----------------------------
// Dumping data to a form

let Show(x) = textB.Text <- sprintf "%A" x
let Clear() = textB.Clear()
let Append(x) = textB.AppendText(x + "\n")

// OK, let's pipeline some data to the output...

(1,2,3) |> Show

[ for i in 0 .. 100 -> (i,i*i) ] |> Show

// ----------------------------
// Scan RSS for news titles

open System.Xml
open System.Collections
open System.Collections.Generic

let xdoc = new XmlDocument()

nytRSS |> Show

xdoc.LoadXml(nytRSS)

// Query the loaded XML
[ for n in xdoc.SelectNodes("//title") -> n.InnerText ] |> Show


// ----------------------------
// Search for URLs in HTML

bbc |> Show

open System.Text.RegularExpressions

let linkPat = "href=\s*\"[^\"h]*(http://[^&\"]*)\""

let bbcLinks = Regex.Matches(bbc,linkPat)

// Fetch out the matched strings
seq { for m in bbcLinks -> m.Groups.Item(1).Value } |> Show

// Put that in a function
let GetLinks (txt:string) =  
   seq { for m in Regex.Matches(txt,linkPat) -> m.Groups.Item(1).Value }

// Collect the links
let CollectLinks url = GetLinks (try http(url) with _ -> "")

CollectLinks "http://www.wired.com" |> Show
CollectLinks "http://news.google.com" |> Show

// ----------------------------
// Crawling (Synchronous)

let limit = 10


let rec Crawl (visited: Set<string>) url =
 
    // show the URL...
    Append (url + "\n")

    // DoEvents() needed for a synchronous crawl when running in the GUI thread
    Application.DoEvents(); 

    // are we done?
    if visited.Count >= limit || visited.Contains(url) 
    then visited
    
    else 
        // get the results and crawl...
        let links = CollectLinks url 
        let visited = visited.Add(url)
        Seq.fold Crawl visited links
    
    
Clear()

Crawl Set.empty "http://news.google.com"
